{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.68631697,  0.72730255, -0.00594355], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pendulum-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAk9ElEQVR4nO3dfXRU9YH/8c+dTDIhhJkQIBMiidBCxRyEKo+jZ1fPkpJq6iPbYy3HRUttxfAkXc+KCq5294Qj29r6ANZ1FbtVqHTFVlZ0s4CxSgSMoOHBaCuSKEyCYGYSJJOH+e4f/Jifg6AJZDLfhPfrnDnH3PudfL9zhby5MzczjjHGCAAAC7mSvQAAAE6FSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArJW0SD366KMaPny40tPTNXnyZG3dujVZSwEAWCopkfr973+vhQsX6t5779Xbb7+tcePGqbi4WA0NDclYDgDAUk4y3mB28uTJmjhxoh555BFJUjQaVX5+vubOnas777yzp5cDALCUu6cnbG1tVVVVlRYtWhTb5nK5VFRUpMrKypPeJxKJKBKJxL6ORqM6fPiwBg0aJMdxEr5mAED3MsaoqalJeXl5crlO/aRej0fq008/VUdHh/x+f9x2v9+v995776T3KSsr03333dcTywMA9KC6ujoNGzbslPt7PFKnY9GiRVq4cGHs61AopIKCAtXV1cnr9SZxZQCA0xEOh5Wfn68BAwZ85bgej9TgwYOVkpKi+vr6uO319fXKzc096X08Ho88Hs+Xtnu9XiIFAL3Y171k0+NX96WlpWn8+PHasGFDbFs0GtWGDRsUCAR6ejkAAIsl5em+hQsXaubMmZowYYImTZqkX/3qVzpy5IhuvvnmZCwHAGCppETq+uuv18GDB7VkyRIFg0F9+9vf1ssvv/yliykAAGe3pPye1JkKh8Py+XwKhUK8JgUAvVBnf47z3n0AAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArNXlSL322mu68sorlZeXJ8dx9MILL8TtN8ZoyZIlGjp0qPr166eioiJ98MEHcWMOHz6sGTNmyOv1KisrS7NmzVJzc/MZPRAAQN/T5UgdOXJE48aN06OPPnrS/Q888IAeeughPfbYY9qyZYv69++v4uJitbS0xMbMmDFDu3btUnl5udatW6fXXntNP/nJT07/UQAA+iZzBiSZtWvXxr6ORqMmNzfXLFu2LLatsbHReDwes2rVKmOMMbt37zaSzLZt22Jj1q9fbxzHMZ988kmn5g2FQkaSCYVCZ7J8AECSdPbneLe+JrV3714Fg0EVFRXFtvl8Pk2ePFmVlZWSpMrKSmVlZWnChAmxMUVFRXK5XNqyZctJv28kElE4HI67AQD6vm6NVDAYlCT5/f647X6/P7YvGAwqJycnbr/b7VZ2dnZszInKysrk8/lit/z8/O5cNgDAUr3i6r5FixYpFArFbnV1dcleEgCgB3RrpHJzcyVJ9fX1cdvr6+tj+3Jzc9XQ0BC3v729XYcPH46NOZHH45HX6427AQD6vm6N1IgRI5Sbm6sNGzbEtoXDYW3ZskWBQECSFAgE1NjYqKqqqtiYjRs3KhqNavLkyd25HABAL+fu6h2am5v1l7/8Jfb13r17tWPHDmVnZ6ugoEALFizQv/zLv2jUqFEaMWKEFi9erLy8PF1zzTWSpPPPP1/f/e53dcstt+ixxx5TW1ub5syZox/84AfKy8vrtgcGAOgDunrZ4KZNm4ykL91mzpxpjDl2GfrixYuN3+83Ho/HTJ061dTU1MR9j0OHDpkbbrjBZGZmGq/Xa26++WbT1NTU7ZcuAgDs1Nmf444xxiSxkaclHA7L5/MpFArx+hQA9EKd/TneK67uAwCcnYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFjLnewFACeKtraqZf9+hbZuVfjdd9V28KDkOPIMHSrvuHHyTZyotJwcuVJTk71UAAlGpGCVyMGD+vTll3Xw5ZfV0dQUv2//foWrqlT/wgvKufpq9SsoUL8RI5Q6cKAcx0nSigEkEpGCFYwxam1o0Ce/+50aX39dpqPjlGPbDh/WgWeflSs9XRkjR2rEP/6jUjIyCBXQB/GaFKzQ0dysj598Up/9+c9fGajjopGI2kMhhauq9GFZmSL19TLG9MBKAfQkIgUrNLz4ohorK6VotMv3bXr3Xe17+GEdra0lVEAfQ6SQdK2HDunQpk1n9D2aq6tVu3y5IgcOECqgDyFSSLqGP/1JrQ0NZ/x9juzZo72/+IWOvPeezGmckQGwD5FC0kXb2qRuOvv5/IMPVPub36hp1y7OqIA+gEihzzn64Yeqe/xxNe/axRkV0MsRKfRJLfv26YMlS3Rg9epjZ2oAeiUihT7LtLfrwOrVqn/++WQvBcBpIlJIuuy/+Ru5Bw5M2Pc/8NxzOvDcc536/SsAdiFSSLr+o0crc/TohH1/09am/c88owO//72i7e0JmwdA9yNSSDrHcTTsppvUb/jwxE1izLGn/v7rvxI3B4BuR6RghTS/XzlXXSXHndi3kzzw3HOcUQG9CJGCFRyXS4OmTtWwH/9YrrS0hM1z/Km/IKECegUiBWs4jqOcK65Q3syZCZ/rwHPPKbhmDaECLMdHdcA6OZdfLhmjT377W5nW1sRMYoyCf/iDopGIhn7/+3LxUR+AlTiTgnUct1s5V16pYT/6kZwEfvquaWtT/fPP6+Pf/lYdzc0JmwfA6SNSsJLjOBpy+eU6pwee+vv05Ze1/5ln1BYK8X5/gGWIFKx1/DWqkffdp4xvfjNxExmjT195RR8/+aTaDh0iVIBFiBSs5rjd8l14oQrmzFH/RP7Cb0eHDm/apLr/+A+1HjyYsHkAdA2RQq+Q8Y1vKP+WW5RZWJjQeRo3b9YnK1eqhQ9PBKxApNArOI6jjJEjde68efJedFHiJjJGn23erNoVK3Tk/fcJFZBkRAq9huM4Ss/L07nz5sk3aVLiJopG1bRjhz5YvFjht95K3DwAvhaRQq+TOnCg8n/6U2VdfHFC54m2tGjfI48o9NZbnFEBSdKlSJWVlWnixIkaMGCAcnJydM0116impiZuTEtLi0pLSzVo0CBlZmZq+vTpqq+vjxtTW1urkpISZWRkKCcnR3fccYfa+c1/dJLjOEobPFjDFyxQ9mWXJXSuts8+01/LyhTevj2h8wA4uS5FqqKiQqWlpXrzzTdVXl6utrY2TZs2TUeOHImNuf322/Xiiy9qzZo1qqio0P79+3XdddfF9nd0dKikpEStra3avHmznn76aa1cuVJLlizpvkeFPs9xHKWkp+vcuXMTHirT1qa9y5apcevWhM4D4MsccwbPYxw8eFA5OTmqqKjQ3/7t3yoUCmnIkCF69tln9fd///eSpPfee0/nn3++KisrNWXKFK1fv17f+973tH//fvn9fknSY489pn/6p3/SwYMHldaJNxcNh8Py+XwKhULyer2nu3z0Ee3Nzap7/HEdrqiQEvi0nDsrS8PnzZN3/HjeQgk4Q539OX5Gr0mFQiFJUnZ2tiSpqqpKbW1tKioqio0ZPXq0CgoKVFlZKUmqrKzUBRdcEAuUJBUXFyscDmvXrl0nnScSiSgcDsfdgOPcmZkaPn9+ws+o2hsb9dd//Vee+gN60GlHKhqNasGCBbrkkks0ZswYSVIwGFRaWpqysrLixvr9fgWDwdiYLwbq+P7j+06mrKxMPp8vdsvPzz/dZaOPclJSdO5ttyn70ksTOo/p6NDeX/xCn735pkw0mtC5AJxBpEpLS7Vz506tXr26O9dzUosWLVIoFIrd6urqEj4neh+Xx6P8n/5U2ZddltA3pu1oalLtI4/os8pKmY6OhM0D4DQjNWfOHK1bt06bNm3SsGHDYttzc3PV2tqqxsbGuPH19fXKzc2NjTnxar/jXx8fcyKPxyOv1xt3A07GnZmp4QsWaPi8eUodNChh87SHw9q7bJkOVVQomqiPEwHQtUgZYzRnzhytXbtWGzdu1IgRI+L2jx8/XqmpqdqwYUNsW01NjWpraxUIBCRJgUBA1dXVamhoiI0pLy+X1+tVYYLf8gZnB8fl0sBLLtGwm25S2glPLXeraFQfP/GEPv3f/1U0EkncPMBZrEsfelhaWqpnn31Wf/zjHzVgwIDYa0g+n0/9+vWTz+fTrFmztHDhQmVnZ8vr9Wru3LkKBAKaMmWKJGnatGkqLCzUjTfeqAceeEDBYFD33HOPSktL5fF4uv8R4qzkuN3KuuQSub1effTQQ2o7dCgh83Q0N8c+nHHIFVfI1YmrUwF0XpcuQT/VZbdPPfWUbrrpJknHfpn3Zz/7mVatWqVIJKLi4mItX7487qm8ffv2afbs2Xr11VfVv39/zZw5U0uXLpXb3blmcgk6OssYoyM1Ndr38MNqSeRrmS6X8mbM0JDLL5c7MzNx8wB9RGd/jp/R70klC5FCV5hoVM3vvafa5cvVUlubsHlcGRnK+d735L/2Wrn790/YPEBf0CO/JwX0Bo7LpczRozVy8WJ58vISNk/0889V/8ILxz7l97PPEjYPcDYhUjgrOC6X0nJyNOrnP1fGqFEJm8e0turgunV6f/FiHd23L2HzAGcLIoWzhuM48gwZouG3365+J1yZ2t1aamu198EHFWlo4B3UgTNApHDW6TdsmEbdf7/SzzknofMc/fBDvX/33Qm7shA4GxApnJVSfT6N/Od/Vv9vfSuh87TW1+svP/+5jn70UULnAfoqIoWzlsfv1/Dbb1f6uecmdJ6je/dq7y9/qdZPP03oPEBfRKRwVks/5xydt3Spcq6+Wq709ITNc/Sjj1Rz552KnPCWYAC+GpHCWc/dv7+GXn+9/NdeK1e/fgmbp7WhQX8tK9PnH33ExRRAJxEpQMfemNZ/1VUaev31UkpKwuY5+uGH2rtsmSL19YQK6AQiBfw/Kf37K+fKK5V/yy1KGTAgYfO01NXpg7vv1ud/+Qsf9QF8DSIFfIErNVWDi4o07OablZLA9+BrPXhQH/3qV2qqrubDE4GvQKSAE7jS0pR92WU6d968hH54YktdnfY9/LCad+8mVMApECngJFxut7ImTdI377oroR+e2HrwoPYuW6bGLVv48ETgJIgUcAqOyyXf+PEaPm+eUhL4ruZtn32mj598Up+98QZnVMAJiBTwNbwXXqhv3nWXXAn8UM7W+np9/OSTOvzaazLt7QmbB+htiBTQCZljxuibd9+t1MGDEzZHeyikun//dx1Ys0bt4XDC5gF6EyIFdILjOPJ++9saPm9eQn/ht6OpSQdWrdK+FSu4PB0QkQK6ZMC4cRp5zz1ye72SK3F/fRrfeEMf/tu/qePo0YTNAfQGRAroAsdxlDlmjM5btkwDL7kk4aHa99BDam9qStgcgO2IFNBFjuMofehQnTNzpgZefHFC5/rsjTe0b/lyrvrDWYtIAacpbcgQDfvxjzXoO9+R43YnbJ7GzZv14bJlag+Heb8/nHUS9zcL6OMcx1FadraG3XSTUjIy1PDii1IizniMUeMbb6ijuVlDb7hBmeefL8dxun8ewEKcSQFnyD1ggIb+4AfKueqqhM7T9M47ql2xQs07d3JGhbMGkQK6QUpGhvJ++EPlzZghJy0tYfO07Nun2scfV/OePYQKZwUiBXQDx3GUkp6u3O9/X3k//KF0Bk/HfXLkiNbV1WnVhx/qf/fv15G2trj9Lfv26cCqVepobj7TZQPW4zUpoBs5Lpf8V18t09Gh/b/7ndSFsx1jjPY2N+ve7dv1UXOzWjo65E1N1ZiBA/VvEycq9QuXuze9844+/+tfNWDcOF6fQp/GmRTQzZyUFOVed53OuemmLj3192Fzs2554w3tCYV0tKNDRlKorU1vNDRo/pYtOtTSEje+9rHHunnlgH2IFJAATkqKcq+99thTf530q127FDrhqb3jtn76qcr374/bZk4xFuhLiBSQQP6rrlLejTcm9J0pgL6M16SABHLcbuVOny6X262G9evVGgwme0lAr8I/74AEc1wu+a+9VsPnz1e/b3zjlONK8vOVeoqLIIZnZmpsdnailghYi0gBPSTz/PNVcOut6n/eeSfdX5yXp3svvFDpKSmxv5gpjqNBHo9+MXGiCrOy4sb7p09P7IIBC/B0H9BDHJdL/c87T8MXLtS+Rx5Rc3V1/H7HUXFenoZlZGjdxx/rUEuLhmdm6voRIzTohE8F9gwdqoGBAJefo88jUkAPchxHntxcnXvbbap7/HGFt2//0v4xAwdqzMCBp/we7oEDlXfjjcc+0wro43i6D+hhjuPIk5engtJS+SZMkFyuTr+LekpmpoZef72yJk2Sk5KS4JUCyceZFJAEjuMobcgQfePOO/XJf/6nXOnpOrxpk1obGk4+PiVF6fn58l97rbIvu4yn+XDWIFJAkjiOIyctTfmzZikaicg7dqw+27xZzbt2KRIMKhqJKCUzU+nDhsk3YYJ8EyaoX0EBgcJZhUgBFnB5PMocM0b9v/UtdRw9KtPeLhONyklJkSs1Va6MDLkS+MGKgK34Uw9YwnEcOR6PXCdcyQeczbhwAgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpditSKFSs0duxYeb1eeb1eBQIBrV+/Pra/paVFpaWlGjRokDIzMzV9+nTV19fHfY/a2lqVlJQoIyNDOTk5uuOOO9Te3t49jwYA0Kd0KVLDhg3T0qVLVVVVpbfeekt/93d/p6uvvlq7du2SJN1+++168cUXtWbNGlVUVGj//v267rrrYvfv6OhQSUmJWltbtXnzZj399NNauXKllixZ0r2PCgDQN5gzNHDgQPPEE0+YxsZGk5qaatasWRPbt2fPHiPJVFZWGmOMeemll4zL5TLBYDA2ZsWKFcbr9ZpIJNLpOUOhkJFkQqHQmS4fAJAEnf05ftqvSXV0dGj16tU6cuSIAoGAqqqq1NbWpqKiotiY0aNHq6CgQJWVlZKkyspKXXDBBfL7/bExxcXFCofDsbOxk4lEIgqHw3E3AEDf1+VIVVdXKzMzUx6PR7feeqvWrl2rwsJCBYNBpaWlKSsrK2683+9XMBiUJAWDwbhAHd9/fN+plJWVyefzxW75+fldXTYAoBfqcqTOO+887dixQ1u2bNHs2bM1c+ZM7d69OxFri1m0aJFCoVDsVldXl9D5AAB2cHf1DmlpaRo5cqQkafz48dq2bZt+/etf6/rrr1dra6saGxvjzqbq6+uVm5srScrNzdXWrVvjvt/xq/+OjzkZj8cjj8fT1aUCAHq5M/49qWg0qkgkovHjxys1NVUbNmyI7aupqVFtba0CgYAkKRAIqLq6Wg0NDbEx5eXl8nq9KiwsPNOlAAD6mC6dSS1atEiXX365CgoK1NTUpGeffVavvvqqXnnlFfl8Ps2aNUsLFy5Udna2vF6v5s6dq0AgoClTpkiSpk2bpsLCQt1444164IEHFAwGdc8996i0tJQzJQDAl3QpUg0NDfqHf/gHHThwQD6fT2PHjtUrr7yi73znO5KkBx98UC6XS9OnT1ckElFxcbGWL18eu39KSorWrVun2bNnKxAIqH///po5c6buv//+7n1UAIA+wTHGmGQvoqvC4bB8Pp9CoZC8Xm+ylwMA6KLO/hznvfsAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWOuMIrV06VI5jqMFCxbEtrW0tKi0tFSDBg1SZmampk+frvr6+rj71dbWqqSkRBkZGcrJydEdd9yh9vb2M1kKAKAPOu1Ibdu2Tb/5zW80duzYuO233367XnzxRa1Zs0YVFRXav3+/rrvuutj+jo4OlZSUqLW1VZs3b9bTTz+tlStXasmSJaf/KAAAfZM5DU1NTWbUqFGmvLzcXHrppWb+/PnGGGMaGxtNamqqWbNmTWzsnj17jCRTWVlpjDHmpZdeMi6XywSDwdiYFStWGK/XayKRSKfmD4VCRpIJhUKns3wAQJJ19uf4aZ1JlZaWqqSkREVFRXHbq6qq1NbWFrd99OjRKigoUGVlpSSpsrJSF1xwgfx+f2xMcXGxwuGwdu3addL5IpGIwuFw3A0A0Pe5u3qH1atX6+2339a2bdu+tC8YDCotLU1ZWVlx2/1+v4LBYGzMFwN1fP/xfSdTVlam++67r6tLBQD0cl06k6qrq9P8+fP1zDPPKD09PVFr+pJFixYpFArFbnV1dT02NwAgeboUqaqqKjU0NOiiiy6S2+2W2+1WRUWFHnroIbndbvn9frW2tqqxsTHufvX19crNzZUk5ebmfulqv+NfHx9zIo/HI6/XG3cDAPR9XYrU1KlTVV1drR07dsRuEyZM0IwZM2L/nZqaqg0bNsTuU1NTo9raWgUCAUlSIBBQdXW1GhoaYmPKy8vl9XpVWFjYTQ8LANAXdOk1qQEDBmjMmDFx2/r3769BgwbFts+aNUsLFy5Udna2vF6v5s6dq0AgoClTpkiSpk2bpsLCQt1444164IEHFAwGdc8996i0tFQej6ebHhYAoC/o8oUTX+fBBx+Uy+XS9OnTFYlEVFxcrOXLl8f2p6SkaN26dZo9e7YCgYD69++vmTNn6v777+/upQAAejnHGGOSvYiuCofD8vl8CoVCvD4FAL1QZ3+O8959AABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABruZO9gNNhjJEkhcPhJK8EAHA6jv/8Pv7z/FR6ZaQOHTokScrPz0/ySgAAZ6KpqUk+n++U+3tlpLKzsyVJtbW1X/ngznbhcFj5+fmqq6uT1+tN9nKsxXHqHI5T53CcOscYo6amJuXl5X3luF4ZKZfr2EtpPp+PPwSd4PV6OU6dwHHqHI5T53Ccvl5nTjK4cAIAYC0iBQCwVq+MlMfj0b333iuPx5PspViN49Q5HKfO4Th1Dsepeznm667/AwAgSXrlmRQA4OxApAAA1iJSAABrESkAgLV6ZaQeffRRDR8+XOnp6Zo8ebK2bt2a7CX1qNdee01XXnml8vLy5DiOXnjhhbj9xhgtWbJEQ4cOVb9+/VRUVKQPPvggbszhw4c1Y8YMeb1eZWVladasWWpubu7BR5FYZWVlmjhxogYMGKCcnBxdc801qqmpiRvT0tKi0tJSDRo0SJmZmZo+fbrq6+vjxtTW1qqkpEQZGRnKycnRHXfcofb29p58KAm1YsUKjR07NvaLp4FAQOvXr4/t5xid3NKlS+U4jhYsWBDbxrFKENPLrF692qSlpZknn3zS7Nq1y9xyyy0mKyvL1NfXJ3tpPeall14yd999t3n++eeNJLN27dq4/UuXLjU+n8+88MIL5p133jFXXXWVGTFihDl69GhszHe/+10zbtw48+abb5o///nPZuTIkeaGG27o4UeSOMXFxeapp54yO3fuNDt27DBXXHGFKSgoMM3NzbExt956q8nPzzcbNmwwb731lpkyZYq5+OKLY/vb29vNmDFjTFFRkdm+fbt56aWXzODBg82iRYuS8ZAS4k9/+pP57//+b/P++++bmpoac9ddd5nU1FSzc+dOYwzH6GS2bt1qhg8fbsaOHWvmz58f286xSoxeF6lJkyaZ0tLS2NcdHR0mLy/PlJWVJXFVyXNipKLRqMnNzTXLli2LbWtsbDQej8esWrXKGGPM7t27jSSzbdu22Jj169cbx3HMJ5980mNr70kNDQ1GkqmoqDDGHDsmqampZs2aNbExe/bsMZJMZWWlMebYPwZcLpcJBoOxMStWrDBer9dEIpGefQA9aODAgeaJJ57gGJ1EU1OTGTVqlCkvLzeXXnppLFIcq8TpVU/3tba2qqqqSkVFRbFtLpdLRUVFqqysTOLK7LF3714Fg8G4Y+Tz+TR58uTYMaqsrFRWVpYmTJgQG1NUVCSXy6UtW7b0+Jp7QigUkvT/35y4qqpKbW1tccdp9OjRKigoiDtOF1xwgfx+f2xMcXGxwuGwdu3a1YOr7xkdHR1avXq1jhw5okAgwDE6idLSUpWUlMQdE4k/T4nUq95g9tNPP1VHR0fc/2RJ8vv9eu+995K0KrsEg0FJOukxOr4vGAwqJycnbr/b7VZ2dnZsTF8SjUa1YMECXXLJJRozZoykY8cgLS1NWVlZcWNPPE4nO47H9/UV1dXVCgQCamlpUWZmptauXavCwkLt2LGDY/QFq1ev1ttvv61t27Z9aR9/nhKnV0UKOB2lpaXauXOnXn/99WQvxUrnnXeeduzYoVAopD/84Q+aOXOmKioqkr0sq9TV1Wn+/PkqLy9Xenp6spdzVulVT/cNHjxYKSkpX7pipr6+Xrm5uUlalV2OH4evOka5ublqaGiI29/e3q7Dhw/3ueM4Z84crVu3Tps2bdKwYcNi23Nzc9Xa2qrGxsa48Scep5Mdx+P7+oq0tDSNHDlS48ePV1lZmcaNG6df//rXHKMvqKqqUkNDgy666CK53W653W5VVFTooYcektvtlt/v51glSK+KVFpamsaPH68NGzbEtkWjUW3YsEGBQCCJK7PHiBEjlJubG3eMwuGwtmzZEjtGgUBAjY2Nqqqqio3ZuHGjotGoJk+e3ONrTgRjjObMmaO1a9dq48aNGjFiRNz+8ePHKzU1Ne441dTUqLa2Nu44VVdXxwW9vLxcXq9XhYWFPfNAkiAajSoSiXCMvmDq1Kmqrq7Wjh07YrcJEyZoxowZsf/mWCVIsq/c6KrVq1cbj8djVq5caXbv3m1+8pOfmKysrLgrZvq6pqYms337drN9+3Yjyfzyl78027dvN/v27TPGHLsEPSsry/zxj3807777rrn66qtPegn6hRdeaLZs2WJef/11M2rUqD51Cfrs2bONz+czr776qjlw4EDs9vnnn8fG3HrrraagoMBs3LjRvPXWWyYQCJhAIBDbf/yS4WnTppkdO3aYl19+2QwZMqRPXTJ85513moqKCrN3717z7rvvmjvvvNM4jmP+53/+xxjDMfoqX7y6zxiOVaL0ukgZY8zDDz9sCgoKTFpampk0aZJ58803k72kHrVp0yYj6Uu3mTNnGmOOXYa+ePFi4/f7jcfjMVOnTjU1NTVx3+PQoUPmhhtuMJmZmcbr9Zqbb77ZNDU1JeHRJMbJjo8k89RTT8XGHD161Nx2221m4MCBJiMjw1x77bXmwIEDcd/no48+Mpdffrnp16+fGTx4sPnZz35m2traevjRJM6PfvQjc+6555q0tDQzZMgQM3Xq1FigjOEYfZUTI8WxSgw+qgMAYK1e9ZoUAODsQqQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1/g9wXqhVHiNSvAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1.4742],\n",
       "         [1.5290]], grad_fn=<MulBackward0>),\n",
       " tensor([[0.4182],\n",
       "         [0.8354]], grad_fn=<NegBackward0>))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class ModelAction(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.fc_state = torch.nn.Sequential(\n",
    "            torch.nn.Linear(3, 128),\n",
    "            torch.nn.ReLU(),\n",
    "        )\n",
    "        self.fc_mu = torch.nn.Linear(128, 1)\n",
    "        self.fc_std = torch.nn.Sequential(\n",
    "            torch.nn.Linear(128, 1),\n",
    "            torch.nn.Softplus(),\n",
    "        )\n",
    "\n",
    "    def forward(self, state):\n",
    "        #[b, 3] -> [b, 128]\n",
    "        state = self.fc_state(state)\n",
    "\n",
    "        #[b, 128] -> [b, 1]\n",
    "        mu = self.fc_mu(state)\n",
    "\n",
    "        #[b, 128] -> [b, 1]\n",
    "        std = self.fc_std(state)\n",
    "\n",
    "        #根据mu和std定义b个正态分布\n",
    "        dist = torch.distributions.Normal(mu, std)\n",
    "\n",
    "        #采样b个样本\n",
    "        #这里用的是rsample,表示重采样,其实就是先从一个标准正态分布中采样,然后乘以标准差,加上均值\n",
    "        sample = dist.rsample()\n",
    "\n",
    "        #样本压缩到-1,1之间,求动作\n",
    "        action = torch.tanh(sample)\n",
    "\n",
    "        #求概率对数\n",
    "        log_prob = dist.log_prob(sample)\n",
    "\n",
    "        #这个值描述动作的熵\n",
    "        entropy = log_prob - (1 - action.tanh()**2 + 1e-7).log()\n",
    "        entropy = -entropy\n",
    "\n",
    "        return action * 2, entropy\n",
    "\n",
    "\n",
    "model_action = ModelAction()\n",
    "\n",
    "model_action(torch.randn(2, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0656],\n",
       "        [ 0.0050]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class ModelValue(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.sequential = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 1),\n",
    "        )\n",
    "\n",
    "    def forward(self, state, action):\n",
    "        #[b, 3+1] -> [b, 4]\n",
    "        state = torch.cat([state, action], dim=1)\n",
    "\n",
    "        #[b, 4] -> [b, 1]\n",
    "        return self.sequential(state)\n",
    "\n",
    "\n",
    "model_value = ModelValue()\n",
    "model_value_next = ModelValue()\n",
    "model_value_next.load_state_dict(model_value.state_dict())\n",
    "\n",
    "model_value(torch.randn(2, 3), torch.randn(2, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1.8301947116851807"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def get_action(state):\n",
    "    state = torch.FloatTensor(state).reshape(1, 3)\n",
    "    action, _ = model_action(state)\n",
    "    return action.item()\n",
    "\n",
    "\n",
    "get_action([1, 2, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200,\n",
       " (array([-0.7754088 , -0.6314596 , -0.01864797], dtype=float32),\n",
       "  1.0557578802108765,\n",
       "  -6.043692695096031,\n",
       "  array([-0.7858418, -0.6184276, -0.333879 ], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#样本池\n",
    "datas = []\n",
    "\n",
    "\n",
    "#向样本池中添加N条数据,删除M条最古老的数据\n",
    "def update_data():\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        next_state, reward, over, _ = env.step([action])\n",
    "\n",
    "        #记录数据样本\n",
    "        datas.append((state, action, reward, next_state, over))\n",
    "\n",
    "        #更新游戏状态,开始下一个动作\n",
    "        state = next_state\n",
    "\n",
    "    #数据上限,超出时从最古老的开始删除\n",
    "    while len(datas) > 100000:\n",
    "        datas.pop(0)\n",
    "\n",
    "\n",
    "update_data()\n",
    "\n",
    "len(datas), datas[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2372/1710091499.py:7: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[-0.9537,  0.3007,  0.3416],\n",
       "         [-0.9814, -0.1921, -0.1296],\n",
       "         [-0.9942, -0.1071,  0.0899],\n",
       "         [-0.9952, -0.0978,  0.0211],\n",
       "         [-0.9945, -0.1048,  0.1593]]),\n",
       " tensor([[ 1.4174],\n",
       "         [-1.7115],\n",
       "         [ 1.5697],\n",
       "         [ 0.2318],\n",
       "         [-0.8715]]),\n",
       " tensor([[-8.0573],\n",
       "         [-8.6968],\n",
       "         [-9.2099],\n",
       "         [-9.2640],\n",
       "         [-9.2241]]),\n",
       " tensor([[-0.9647,  0.2633,  0.7798],\n",
       "         [-0.9861, -0.1661, -0.5305],\n",
       "         [-0.9929, -0.1193,  0.2450],\n",
       "         [-0.9953, -0.0969, -0.0175],\n",
       "         [-0.9948, -0.1023, -0.0501]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0]]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取一批数据样本\n",
    "def get_sample():\n",
    "    #从样本池中采样\n",
    "    samples = random.sample(datas, 64)\n",
    "\n",
    "    #[b, 3]\n",
    "    state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    action = torch.FloatTensor([i[1] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 1]\n",
    "    reward = torch.FloatTensor([i[2] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 3]\n",
    "    next_state = torch.FloatTensor([i[3] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    over = torch.LongTensor([i[4] for i in samples]).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "state[:5], action[:5], reward[:5], next_state[:5], over[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1337.0967746729043"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step([action])\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play and random.random() < 0.2:  #跳帧\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def soft_update(model, model_next):\n",
    "    for param, param_next in zip(model.parameters(), model_next.parameters()):\n",
    "        #以一个小的比例更新\n",
    "        value = param_next.data * 0.995 + param.data * 0.005\n",
    "        param_next.data.copy_(value)\n",
    "\n",
    "\n",
    "soft_update(torch.nn.Linear(4, 64), torch.nn.Linear(4, 64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 1])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_target(reward, next_state, over):\n",
    "    #首先使用model_action计算动作和熵\n",
    "    #[b, 4] -> [b, 1],[b, 1]\n",
    "    action, entropy = model_action(next_state)\n",
    "\n",
    "    #评估next_state的价值\n",
    "    #[b, 4],[b, 1] -> [b, 1]\n",
    "    target = model_value_next(next_state, action)\n",
    "\n",
    "    #这里的操作是在target上加上了动作的熵\n",
    "    #[b, 1] - [b, 1] -> [b, 1]\n",
    "    target += 0.005 * entropy\n",
    "\n",
    "    #[b, 1]\n",
    "    target *= 0.99\n",
    "    target *= (1 - over)\n",
    "    target += reward\n",
    "\n",
    "    return target\n",
    "\n",
    "\n",
    "get_target(reward, next_state, over).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-0.0040, grad_fn=<MeanBackward0>)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_loss_action(state):\n",
    "    #计算action和熵\n",
    "    #[b, 3] -> [b, 1],[b, 1]\n",
    "    action, entropy = model_action(state)\n",
    "\n",
    "    #使用value网络评估action的价值\n",
    "    #[b, 3],[b, 1] -> [b, 1]\n",
    "    value = model_value(state, action)\n",
    "\n",
    "    #熵,这个值期望的是越大越好,但是这里是计算loss,所以符号取反\n",
    "    #[1] - [b, 1] -> [b, 1]\n",
    "    loss_action = -0.005 * entropy\n",
    "\n",
    "    #减去value,所以value越大越好,这样loss就会越小\n",
    "    loss_action -= value\n",
    "\n",
    "    return loss_action.mean()\n",
    "\n",
    "\n",
    "get_loss_action(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "OHoSU6uI-xIt",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 400 -1661.8095294373827\n",
      "10 2400 -1514.7647813052058\n",
      "20 4400 -1575.5831378945663\n",
      "30 6400 -662.5823267235334\n",
      "40 8400 -986.4176449379411\n",
      "50 10400 -998.4606477021243\n",
      "60 12400 -370.6228748522011\n",
      "70 14400 -225.43343732554158\n",
      "80 16400 -142.0383754240294\n",
      "90 18400 -321.4496607018651\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    optimizer_action = torch.optim.Adam(model_action.parameters(), lr=3e-4)\n",
    "    optimizer_value = torch.optim.Adam(model_value.parameters(), lr=3e-3)\n",
    "\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #训练N次\n",
    "    for epoch in range(100):\n",
    "        #更新N条数据\n",
    "        update_data()\n",
    "\n",
    "        #每次更新过数据后,学习N次\n",
    "        for i in range(200):\n",
    "            #采样一批数据\n",
    "            state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "            #对reward偏移,为了便于训练\n",
    "            reward = (reward + 8) / 8\n",
    "\n",
    "            #计算target,这个target里已经考虑了动作的熵\n",
    "            #[b, 1]\n",
    "            target = get_target(reward, next_state, over)\n",
    "            target = target.detach()\n",
    "\n",
    "            #计算value\n",
    "            value = model_value(state, action)\n",
    "\n",
    "            #计算loss,value的目标是要贴近target\n",
    "            loss_value = loss_fn(value, target)\n",
    "\n",
    "            #更新参数\n",
    "            optimizer_value.zero_grad()\n",
    "            loss_value.backward()\n",
    "            optimizer_value.step()\n",
    "\n",
    "            #使用model_value计算model_action的loss\n",
    "            loss_action = get_loss_action(state)\n",
    "            optimizer_action.zero_grad()\n",
    "            loss_action.backward()\n",
    "            optimizer_action.step()\n",
    "\n",
    "            #增量更新next模型\n",
    "            soft_update(model_value, model_value_next)\n",
    "\n",
    "        if epoch % 10 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(10)]) / 10\n",
    "            print(epoch, len(datas), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAu3UlEQVR4nO3df3RU1b338c+ZzGRISGZCAklMSQQBQQpaBYHR2oqkpJT+sOKzvJSl3NalVxosiPVe8VqoriIue++16rW0t95SulYrfdCCilil/IhFIj8iaMCIWKEgMAkBMxNiMjPJ7OcPyzwEQQOZZHbS92utWcucs2ef72zH+XjO7NnHMcYYAQBgIVeqCwAA4GwIKQCAtQgpAIC1CCkAgLUIKQCAtQgpAIC1CCkAgLUIKQCAtQgpAIC1CCkAgLVSFlJPPvmkBg0apD59+mj8+PHaunVrqkoBAFgqJSH1hz/8QfPmzdPChQv1xhtv6LLLLlNZWZnq6upSUQ4AwFJOKhaYHT9+vK688kr993//tyQpHo+ruLhYd955p+69997uLgcAYCl3dx8wGo2qqqpK8+fPT2xzuVwqLS1VZWXlGZ8TiUQUiUQSf8fjcR0/flx5eXlyHKfLawYAJJcxRo2NjSoqKpLLdfaLet0eUvX19Wpra1NBQUG77QUFBXrnnXfO+JzFixfrgQce6I7yAADd6ODBgxo4cOBZ93d7SJ2P+fPna968eYm/Q6GQSkpKdPDgQfl8vhRWBgA4H+FwWMXFxcrOzv7Udt0eUv3791daWppqa2vbba+trVVhYeEZn+P1euX1ej+x3efzEVIA0IN91lc23T67Lz09XWPGjNG6desS2+LxuNatW6dAINDd5QAALJaSy33z5s3TzJkzNXbsWI0bN04/+9nP1NTUpO9+97upKAcAYKmUhNRNN92ko0ePasGCBQoGg/rCF76gP/3pT5+YTAEA+MeWkt9JdVY4HJbf71coFOI7KQDogTr6Oc7afQAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGsRUgAAaxFSAABrEVIAAGudc0i9+uqr+sY3vqGioiI5jqNVq1a122+M0YIFC3TBBRcoIyNDpaWl2rt3b7s2x48f14wZM+Tz+ZSTk6Nbb71VJ06c6NQLAQD0PuccUk1NTbrsssv05JNPnnH/I488oscff1y/+MUvtGXLFvXt21dlZWVqaWlJtJkxY4Z2796ttWvXavXq1Xr11Vd1++23n/+rAAD0TqYTJJmVK1cm/o7H46awsND89Kc/TWxraGgwXq/XPP3008YYY95++20jyWzbti3R5qWXXjKO45hDhw516LihUMhIMqFQqDPlAwBSpKOf40n9Tmrfvn0KBoMqLS1NbPP7/Ro/frwqKyslSZWVlcrJydHYsWMTbUpLS+VyubRly5Yz9huJRBQOh9s9AAC9X1JDKhgMSpIKCgrabS8oKEjsCwaDys/Pb7ff7XYrNzc30eZ0ixcvlt/vTzyKi4uTWTYAwFI9Ynbf/PnzFQqFEo+DBw+muiQAQDdIakgVFhZKkmpra9ttr62tTewrLCxUXV1du/2tra06fvx4os3pvF6vfD5fuwcAoPdLakgNHjxYhYWFWrduXWJbOBzWli1bFAgEJEmBQEANDQ2qqqpKtFm/fr3i8bjGjx+fzHIAAD2c+1yfcOLECb333nuJv/ft26edO3cqNzdXJSUlmjt3rn7yk59o2LBhGjx4sH70ox+pqKhI119/vSTpkksu0Ve/+lXddttt+sUvfqFYLKbZs2frn/7pn1RUVJS0FwYA6AXOddrghg0bjKRPPGbOnGmM+Xga+o9+9CNTUFBgvF6vmTRpktmzZ0+7Po4dO2amT59usrKyjM/nM9/97ndNY2Nj0qcuAgDs1NHPcccYY1KYkeclHA7L7/crFArx/RQA9EAd/RzvEbP7AAD/mAgpAIC1CCkAgLUIKQCAtQgpAIC1CCkAgLUIKQCAtQgpAIC1CCkAgLUIKQCAtQgpAIC1znkVdACpY+JxxT78UOEdO9T41luKHj0qSUovKFD26NHyX3GF3H6/HBf//4negZACeohYOKwP//IXBZ99VrFjx6RT14bevVvHN25Uel6eCv/P/1G/q6+WOzs7dcUCSUJIAT1A7MMPdeT//l/Vr10rE42euVE8rujRo/rgf/9XLYcOqfDGG+Xx+7u3UCDJuCYAWC4eiejIH/7w6QF1WvujL72k4DPPKB6LdUOFQNchpADLHVu/XkdfeqlDAXWSiUZV98ILOl5R0YWVAV2PkAIsFguHVffii+2/f+qoeFxH16xRa2Nj8gsDugkhBVjs2CuvKHLo0Hk/v3n/fh1bvz6JFQHdi5ACLGXicbWGwzJtbeffR2ur2pqbk1gV0L0IKcBS8ZaWpF2qM+dzuRCwACEFWCre0qLWUKjT/dSuXKnm999PQkVA9yOkAEvFwmG1dOL7qJPizc1MRUePRUgBlmoNhxU5ciTVZQApRUgBvZwrI0OOx5PqMoDzQkgBFjLGKB6JJKUv/9ix8ubnJ6UvoLsRUoCNjPl4EdkkcPt8cqWnJ6UvoLsRUoCNjFGktjYpXbl9PjmEFHooQgqwkInHdaK6OjmdOY4cx0lOX0A3I6QAG8Xj+mjfvlRXAaQcIQX0Yk56Ojc/RI9GSAEWaotEzm/l89N4CwqUfemlSagISA1CCrBQrL4+KSHl8nq5Oy96NEIKsFCktlbJWBLWlZ6utKysJPQEpAYhBVgo/MYbUjyelL4cF/+Zo+fi3QtYqOXgwaRc7gN6OkIK6MW8n/tcqksAOoWQAiwTj0ZlknGpLy1Nuddc0/l+gBQipADLtIbDSVlc1nEcpbOwLHo4QgqwTGsolLQV0D39+iWlHyBVCCnAMk3vvZe0FdCZ2YeejncwYJnWDz9M2pkU0NMRUkAvlTFokMSZFHo43sGARUxbm+KxWFL68o8fz+U+9Hi8gwGLxKNRxUKhpPTlzc/nTAo9Hu9gwCLxSEStSQopd05OUvoBUomQAiwS+/BDnXj77aT05bhc3JEXPR4hBVjERKNqa2xMdRmANQgpoBfyFhbK7fOlugyg0wgpwBLGGLU1Nyelr4zBg+XJzU1KX0AqEVKARSJ1dUnpx+33K61Pn6T0BaQSIQXYwpiPbxufBGmZmXLS05PSF5BKhBRgi3hcx9atS05fjsPMPvQKhBRgkVZm9gHtEFJAL5PWt68yBw1KdRlAUhBSgCWSNbMvLTtbGRddlJS+gFQjpABLROvrk3LbeFd6ujwsiYRegpACLBGtr5eM6XQ/jtuttL59k1ARkHqEFGCJY3/+s0ySbtPBLTrQW/BOBizB3XiBTyKkAAuYJFzmO8k/ZkzS+gJSjZACLGBiMZnW1s535DjKGjmy8/0AliCkAAu0hsNqa2lJSl/pBQVJ6QewASEFWKC1sVHxZPxOynHk8fs73w9giXMKqcWLF+vKK69Udna28vPzdf3112vPnj3t2rS0tKi8vFx5eXnKysrStGnTVFtb267NgQMHNHXqVGVmZio/P1/33HOPWpNxqQPooRqrq9XywQfJ6YyZfehFzundXFFRofLycr3++utau3atYrGYJk+erKampkSbu+66Sy+88IJWrFihiooKHT58WDfccENif1tbm6ZOnapoNKrNmzdr2bJl+s1vfqMFCxYk71UBPU1bW1J+IwX0No7pxLSio0ePKj8/XxUVFfrSl76kUCikAQMG6Pe//71uvPFGSdI777yjSy65RJWVlZowYYJeeuklff3rX9fhw4dV8Pdr57/4xS/0b//2bzp69KjSO3B7gXA4LL/fr1AoJB93H0UPZ4xR3apV+mDp0k735R83Thfdc49cXm8SKgO6Tkc/xzt1XSAUCkmScv9+B9CqqirFYjGVlpYm2owYMUIlJSWqrKyUJFVWVmr06NGJgJKksrIyhcNh7d69+4zHiUQiCofD7R5ArxGPqy1Jv5HKHDJETlpaUvoCbHDeIRWPxzV37lxdffXVGjVqlCQpGAwqPT1dOaetG1ZQUKBgMJhoU3Da7KOTf59sc7rFixfL7/cnHsXFxedbNmCdeCym1oaGpPTlycvjOyn0Ku7zfWJ5ebl27dqlTZs2JbOeM5o/f77mzZuX+DscDhNU6HGMMYrFYopGozp27Jhqamr0zjvv6L09e/R+ZaXqDx1Sc2urxvbvr5lDhyrb4znnY7h9PombHaIXOa+Qmj17tlavXq1XX31VAwcOTGwvLCxUNBpVQ0NDu7Op2tpaFRYWJtps3bq1XX8nZ/+dbHM6r9crL9fY0QOdDKYPPvhANTU1qqio0ObNm7Vr1y41NTV9vNKEMe1WPx+YmXneK1A4Lhd35EWvck4hZYzRnXfeqZUrV2rjxo0aPHhwu/1jxoyRx+PRunXrNG3aNEnSnj17dODAAQUCAUlSIBDQokWLVFdXp/z8fEnS2rVr5fP5NJJfyqMXiUaj2rFjh1auXKnKykpVV1cnvsf1er0aNGiQBg8erKKcHDk7dqhvWpq8aWkanJ2tPnyvBEg6x5AqLy/X73//ez333HPKzs5OfIfk9/uVkZEhv9+vW2+9VfPmzVNubq58Pp/uvPNOBQIBTZgwQZI0efJkjRw5UjfffLMeeeQRBYNB3X///SovL+dsCT2eMUYtLS3asWOHfvnLX2rjxo2qra2V4zjy+XyaPHmyJk6cqKuvvloDBw5URkaGTF2d3v+3f1Oa48j197Og8zkbyhwyRN4LLkj2SwJS6pxCasmSJZKka6+9tt32pUuX6p//+Z8lSY8++qhcLpemTZumSCSisrIy/fznP0+0TUtL0+rVqzVr1iwFAgH17dtXM2fO1IMPPti5VwKk0MnLem+99ZaWLVum3/72tzpx4oRycnI0ceJEXXfddfrGN76hoUOHKu3vZ0mO48gYoxNHjyo9CWdOnrw8ubOzO90PYJNzvtz3Wfr06aMnn3xSTz755FnbXHjhhVqzZs25HBqwWnNzs/7nf/5HS5cuVU1NjTwej8rKyjRjxgxNnDhRhYWFcp1l1l306NGk1JCWlSVXRkZS+gJscd6z+4B/dMYYxeNxvfvuu1q0aJGee+45tba2avjw4br//vs1adIk9evXT67PmMzQ8re/JaWetIwMuTrwY3igJyGkgPNkjNErr7yiH//4x6qqqlJ+fr6mT5+uH/zgByopKZHUge+WjNHRV15JWk3M7ENvQ0gB5+jkGdTzzz+ve++9V++9954uuugiPfTQQyorK2OpLiCJCCngHBhjFI1G9fTTT2vhwoWqra1VaWmpFi1apMsvvzwxKaK7uX0++a+8MiXHBroSIQWcA2OMli9frvvuu09Hjx7V9OnTtXDhQg0ZMuS8+mv76KOkrH7upKcrfcCATvcD2IaQAjooHo9r7dq1euCBB3T06FHdcMMNWrRoUaeW6IodO9ZutYnz5bjd8py2ZibQG7ASJdAB8Xhcmzdv1r333qtDhw5p2rRpeuSRR9otC3Y+Ptq/XyYJN/x00tKUlpXV6X4A2xBSQAcEg0EtWrRI1dXVGjdunBYuXKgLL7yw07PpGjZvlolGk1IjM/vQGxFSwGeIx+N64okntH79eg0YMEAPPfSQRowYkeqy2nG4PQd6Kd7ZwKeIx+Nas2aNnnrqKfXt21cLFizQhAkTknLW0ombYn9C/7KypPUF2ISQAj7FwYMHtWTJEoXDYX3961/XjTfeKM953OfpTEw0KhOLJaUv71lucwP0dIQUcBatra1as2aNKioqVFhYqB/84Afq379/8vpvalJbc3NS+kr/+21vgN6GkALOIhgM6sknn1QkEtGsWbM0ZsyYpE5OiNbWKnb8eFL68vj9SekHsA0hBZzFH//4R73zzju65JJL9J3vfCfp/Tfv36/IkSPJ6YyZfeilCCngDOrr67Vs2TJ5PB7NmDFDhYWF1k7xdtz8Jh+9FyEFnCYej2v16tV6//33dckll2jKlClJmyxxUjJn9vW75hruI4Vei5ACTtPY2KiKigp99NFHmjBhgoYPH578s6h4PGmTJjx5efxOCr0W72zgFMYY7du3T1u3blVGRoauv/56eb3e5B+ntTVpkybSc3PlpGj1daCrEVLAKYwx2rVrl/bu3atBgwbpqquu6pLjtDU3q2nv3qT05c7OTtrEiWg0qmg0mtTLkUBnEFLAKWKxmDZt2qS2tjZde+216tu3b5ccp625WU01NUnrL1krYPz1r39NvH7ABoQUcIpIJKJNmzbJcRyV9YClhhy3W06SJnXE43G99tprevTRR/XRRx8lpU+gswgp4BSHDx/W/v37VVBQoIsvvrhLjpHMS2kZF12kjAsvTEpfjY2Nevnll7Vx40a99957SekT6CxCCjhFTU2NIpGILr74YmVlZXXZb6PaTpxISj/uvn2VloTp58YYHTp0SBs3btRHH32kF198ke+lYAVCCjhFdXW1jDEaNmyYMjMzu+w4kWAwKf2kZWUpLUnfm61cuVLH/z7j8Pnnn1c4HE5Kv0BnEFLAKf7617/KcRwVFxd3ydTzk0Lbt59136GmJq0+eFBPv/++/nz4sJo+ZaV0l9crVxLqDIVCevbZZxN/Hzx4UK+//nqn+wU6i/VUgFMcPnxYHo9H/fv3l7sLlxtqfPPNT2wzxmjfiRNauGOH9p84oZa2Nvk8Ho3q10//ceWV8nTRD3aNMXrttde0f//+xLaGhgZt2rRJEydOVHp6epccF+gIzqSAU4RCIaWnp8vv93f7Wn3vnzih2157TTWhkJrb2mQkhWIxvVZXpzlbtuhYS0v7J7hccmdldfq4LS0tWrlypU6c8j1ZLBbTa6+9pkOHDnW6f6AzCCngFMYYpaenKyMjo9tD6me7dyt0lkt7W+vrtfbw4Xbb0jIz5bviik4fd8+ePdq+fbvi8Xi77W+++ab27t3LBAqkFCEFnMbtdid9QdnTFd9+e6f7cNLS5MnN7VQfbW1t2r59u9577z05jpMIZsdx1NDQoI0bN6q1tbXTtQLni5ACTmOM6dKzB8dxlDF4cOf7cbuV3sk7BZ84cUIrV66U1+vVTTfdpFGjRkmSpkyZovz8fK1atUqNjY2drhU4X4QUcJq2tjbFPmVGXVeZWlwsz1kuMQ7KytKlp501OS5Xp2/RceDAAR0+fFgLFy7Ugw8+qD59+shxHM2ZM0cPPfSQ3G63qqqqOnUMoDOY3QecwnEcRaNRNTc3yxjTZd9Lefx+5U6cqOMbNiS2lRUVSZJ+8uabira1KS4pzXGUk56u/7zySl142iSJgmnTOl3HO++8o8cee0zjxo3T8ePH1dTUJMdxlJ+fr2uuuUYjRozQW2+91enjAOeLkAJO4ff7FY1GFQqFujSkXBkZ6hcIKLR9u9r+fjnNcRyVFRVpYGamVn/wgY61tGhQVpZuGjxYeaf9Fsp7wQXqFwh0ur5vfvObSk9Pl+M4am1tVX19vTIyMuTz+eT1ejVhwgRddtllnToG0BmEFHCKoqIixWIx1dfXq7W1tct+I+Q4jnyXX64BU6ao9tlnZf6+6rjjOBrVr59G9et31ue6+/VT0c03y+3zdbqOkz9YNsaoqalJx48f14ABA5T197M2x3G6dOUN4LPwnRRwiiFDhsgYow8++ECRSKRLj+XyelXw7W8r97rr5HTwh8NpWVm64KablDNuXNJvdLhv3z61trZq4MCB/IAX1uBMCjjF6NGj5TiO9u7dq6amJmVnZ3fp8dIyMzXwe9+TJzdXxzdsULSu7oztnLQ09Sku/jjUrr22Sy5D7tixQ5I0YsSILp+CD3QUIQWc4pJLLpHX69W7776rpqamLv1eSvr4cpq7b19dcOON8l16qT7cvFkndu9WJBhUPBJRWlaW+gwcKP/YsfKPHauMkpIuq+fkLL6RI0cSUrAGIQWcoqioSIMHD1ZNTY3effddXXTRRd1yXJfXq6xRo9T34ovV1tws09oqE4/LSUuTy+ORKzNTri5cS/DDDz9UdXW1+vbtq5EjRyotyZcSgfPFd1LAKbxer774xS/KGKOXX365W4/tOI5cXq88OTlK799f3vx8peflye3zdWlASdL27dtVX1+vIUOG6MILL+z2JaGAsyGkgFN4PB598YtflNvtVkVFhZqbm1NdUpczxmjTpk1qbm7W8OHDNXDgwFSXBCQQUsApHMfRqFGjdPHFF2v//v2qrKxMdUld7tChQ6qsrJTH49HVV1+tnJycVJcEJBBSwCkcx9HgwYM1fvx4NTc3a9WqVV0+FT2VjDF64403VF1drby8PE2ePJlLfbAKIQWcJjs7W9dee60yMzO1efNmvfvuu732dhUtLS3685//rPr6egUCAQ0dOjTVJQHtEFLAaRzH0de+9jUNGTJEb7/9ttasWZOSBWe7w759+/T000+rT58+uuOOO5jVB+sQUsAZ5Obm6pZbblEsFtPvfvc7BYPBXnc21draqiVLluj48eO69tprWaMPViKkgLP49re/reHDh6umpka/+93vUl1OUhljtHXrVj3//PPy+/2aPn16l6+uAZwPQgo4i8LCQs2ePVt9+vTRkiVLtH379l5xNmWMUX19vZ544gkdOXJEpaWl+spXvsKlPliJkALOwu1262tf+5quueYa1dbW6oknnlB9fX2qy+q0trY2rVq1Sq+88or8fr9uv/12DRgwINVlAWdESAGfori4WOXl5fL5fFq9erWeeeaZHj2Jwhij999/X4888ohCoZC+853v6Etf+hLTzmEtQgr4FC6XS1OmTNHtt9+ujz76SA8++KBef/31HnvZLxgM6oc//KH27dunq666Sj/84Q9ZTBZWI6SAz+ByuVReXq5Jkybp6NGjuu+++/T222+nuqxz1tDQoJ/97Gdav369SkpK9K//+q8qKiriLApWI6SADigsLNR9992n0aNHa+vWrVq4cKH27dvXY86oIpGIli5dql/96ldyuVyaO3euSktLmSwB6xFSQAe4XC4FAgE9/PDDKioq0nPPPae7775bf/vb31Jd2mc6ceKEli5dqvvvv1+xWEyzZs3S97///cSt4wGbEVJAB7lcLn3lK1/RggULlJeXpxdeeEH33Xef3nvvvVSXdlaNjY166qmn9MADD0iSZsyYobvvvltpaWlc5kOPwE0PgXPgOI6mT5+ueDyuBx54QH/84x9VX1+vRYsW6YorrrDm8pkxRg0NDfrJT36iX//614pEIrrlllv0wAMPaMCAAQQUegzOpIBz4DiOvF6vZs6cqUcffVQXXnih1q1bp+nTp+vZZ59VOBxOdYmKxWLatm2bbrvtNj3++ONyuVy644479J//+Z/Kz88noNCjcCYFnCPHceR2u3X99dcrIyNDP/7xj/XGG29o7ty5mj59uubMmaPi4uJE2+5wcgJHLBbTsmXLtGTJEu3atUt5eXm66667dNtttykzM5OAQo/jmJ4yPekU4XBYfr9foVBIPp8v1eXgH5QxRvF4XHv37tWiRYv03HPPKRqNaujQobr//vs1adIk5eXlyXGcLg0HY4yampr05ptv6rHHHtPLL7+sSCSiYcOG6T/+4z/05S9/WV6vl4CCVTr6OU5IAUnQ1NSkX/3qV1q6dKlqamqUnp6ua6+9Vt/5znc0ceJEFRQUyOVK7tV1Y4zC4bB27typFStW6JlnntHRo0c1YMAA3XjjjZo9e7aGDx9OOMFKhBTQjYwxisViqq6u1m9/+1stW7ZMjY2N6tevn8aMGaOJEyfqm9/8pi6++OLE5IpzDY+T/6kaY3T8+HG98sorWrVqlaqqqnTgwAGlpaVp/Pjxuvvuu/WlL31Jfr+fgIK1CCkgBYwxamlp0Ztvvqlf/vKX2rBhg4LBoBzHUXZ2ti6//HJdd911uuqqqzRw4EBlZmYqIyNDXq9Xbrc7cbZljFFra6ui0agikYiampr04Ycfqrq6Wi+//LIqKysVDAYVjUaVlZWlz3/+8/qXf/kXffWrX1Vubm7Sz9qAZCOkgBSLRqPasWOHVq5cqcrKSr311luJ2X99+vTR5z73OQ0aNEjFxcXKz8+Xz+dTnz59JH18Q8JwOKzjx4/ryJEj2rdvnw4cOKBQKCRjjDwej4YMGaJAIKDrrrsuEU5AT0FIARYwxigajerQoUOqqalRRUWFNm/erN27d6upqUnGmMRD+v+XAE//23EceTweFRcX64tf/KKuueYafeELX9BFF12k7OxsLuuhx+mSkFqyZImWLFmi/fv3S5I+//nPa8GCBZoyZYokqaWlRXfffbeWL1+uSCSisrIy/fznP1dBQUGijwMHDmjWrFnasGGDsrKyNHPmTC1evFhud8dnwxNS6IlOXsKLxWI6duyYampqtGfPHv31r39VMBhUQ0ODmpubFY/H5fF45PP5NGDAABUXF+uSSy7RyJEjNXDgQGVkZMjtdnf5rEGgK3X0c/ycfic1cOBAPfzwwxo2bJiMMVq2bJm+9a1vaceOHfr85z+vu+66Sy+++KJWrFghv9+v2bNn64YbbtBrr70m6eObrU2dOlWFhYXavHmzjhw5oltuuUUej0cPPfRQ514xYLmTZ0Mej0eZmZkqLi7W5MmTU10WYDfTSf369TNPPfWUaWhoMB6Px6xYsSKxr6amxkgylZWVxhhj1qxZY1wulwkGg4k2S5YsMT6fz0QikQ4fMxQKGUkmFAp1tnwAQAp09HP8vKcAtbW1afny5WpqalIgEFBVVZVisZhKS0sTbUaMGKGSkhJVVlZKkiorKzV69Oh2l//KysoUDoe1e/fusx4rEokoHA63ewAAer9zDqnq6mplZWXJ6/Xqjjvu0MqVKzVy5EgFg0Glp6crJyenXfuCggIFg0FJH98V9NSAOrn/5L6zWbx4sfx+f+JxcskZAEDvds4hNXz4cO3cuVNbtmzRrFmzNHPmzC6/S+n8+fMVCoUSj4MHD3bp8QAAdjjnBWbT09M1dOhQSdKYMWO0bds2PfbYY7rpppsUjUbV0NDQ7myqtrZWhYWFkj6+u+nWrVvb9VdbW5vYdzZer5cbtAHAP6BO/yw9Ho8rEolozJgx8ng8WrduXWLfnj17dODAAQUCAUlSIBBQdXW16urqEm3Wrl0rn8+nkSNHdrYUAEAvc05nUvPnz9eUKVNUUlKixsZG/f73v9fGjRv18ssvy+/369Zbb9W8efOUm5srn8+nO++8U4FAQBMmTJAkTZ48WSNHjtTNN9+sRx55RMFgUPfff7/Ky8s5UwIAfMI5hVRdXZ1uueUWHTlyRH6/X5deeqlefvllfeUrX5EkPfroo3K5XJo2bVq7H/OelJaWptWrV2vWrFkKBALq27evZs6cqQcffDC5rwoA0CuwLBIAoNt19HOcpZIBANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADWIqQAANYipAAA1iKkAADW6lRIPfzww3IcR3Pnzk1sa2lpUXl5ufLy8pSVlaVp06aptra23fMOHDigqVOnKjMzU/n5+brnnnvU2tramVIAAL3QeYfUtm3b9Mtf/lKXXnppu+133XWXXnjhBa1YsUIVFRU6fPiwbrjhhsT+trY2TZ06VdFoVJs3b9ayZcv0m9/8RgsWLDj/VwEA6J3MeWhsbDTDhg0za9euNV/+8pfNnDlzjDHGNDQ0GI/HY1asWJFoW1NTYySZyspKY4wxa9asMS6XywSDwUSbJUuWGJ/PZyKRSIeOHwqFjCQTCoXOp3wAQIp19HP8vM6kysvLNXXqVJWWlrbbXlVVpVgs1m77iBEjVFJSosrKSklSZWWlRo8erYKCgkSbsrIyhcNh7d69+4zHi0QiCofD7R4AgN7Pfa5PWL58ud544w1t27btE/uCwaDS09OVk5PTbntBQYGCwWCizakBdXL/yX1nsnjxYj3wwAPnWioAoIc7pzOpgwcPas6cOfrd736nPn36dFVNnzB//nyFQqHE4+DBg912bABA6pxTSFVVVamurk5XXHGF3G633G63Kioq9Pjjj8vtdqugoEDRaFQNDQ3tnldbW6vCwkJJUmFh4Sdm+538+2Sb03m9Xvl8vnYPAEDvd04hNWnSJFVXV2vnzp2Jx9ixYzVjxozEP3s8Hq1bty7xnD179ujAgQMKBAKSpEAgoOrqatXV1SXarF27Vj6fTyNHjkzSywIA9Abn9J1Udna2Ro0a1W5b3759lZeXl9h+6623at68ecrNzZXP59Odd96pQCCgCRMmSJImT56skSNH6uabb9YjjzyiYDCo+++/X+Xl5fJ6vUl6WQCA3uCcJ058lkcffVQul0vTpk1TJBJRWVmZfv7znyf2p6WlafXq1Zo1a5YCgYD69u2rmTNn6sEHH0x2KQCAHs4xxphUF3GuwuGw/H6/QqEQ308BQA/U0c9x1u4DAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWIuQAgBYi5ACAFiLkAIAWMud6gLOhzFGkhQOh1NcCQDgfJz8/D75eX42PTKkjh07JkkqLi5OcSUAgM5obGyU3+8/6/4eGVK5ubmSpAMHDnzqi/tHFw6HVVxcrIMHD8rn86W6HGsxTh3DOHUM49Qxxhg1NjaqqKjoU9v1yJByuT7+Ks3v9/Mm6ACfz8c4dQDj1DGMU8cwTp+tIycZTJwAAFiLkAIAWKtHhpTX69XChQvl9XpTXYrVGKeOYZw6hnHqGMYpuRzzWfP/AABIkR55JgUA+MdASAEArEVIAQCsRUgBAKzVI0PqySef1KBBg9SnTx+NHz9eW7duTXVJ3erVV1/VN77xDRUVFclxHK1atardfmOMFixYoAsuuEAZGRkqLS3V3r1727U5fvy4ZsyYIZ/Pp5ycHN166606ceJEN76KrrV48WJdeeWVys7OVn5+vq6//nrt2bOnXZuWlhaVl5crLy9PWVlZmjZtmmpra9u1OXDggKZOnarMzEzl5+frnnvuUWtra3e+lC61ZMkSXXrppYkfngYCAb300kuJ/YzRmT388MNyHEdz585NbGOsuojpYZYvX27S09PNr3/9a7N7925z2223mZycHFNbW5vq0rrNmjVrzL//+7+bP/7xj0aSWblyZbv9Dz/8sPH7/WbVqlXmzTffNN/85jfN4MGDTXNzc6LNV7/6VXPZZZeZ119/3fzlL38xQ4cONdOnT+/mV9J1ysrKzNKlS82uXbvMzp07zde+9jVTUlJiTpw4kWhzxx13mOLiYrNu3Tqzfft2M2HCBHPVVVcl9re2tppRo0aZ0tJSs2PHDrNmzRrTv39/M3/+/FS8pC7x/PPPmxdffNG8++67Zs+ePea+++4zHo/H7Nq1yxjDGJ3J1q1bzaBBg8yll15q5syZk9jOWHWNHhdS48aNM+Xl5Ym/29raTFFRkVm8eHEKq0qd00MqHo+bwsJC89Of/jSxraGhwXi9XvP0008bY4x5++23jSSzbdu2RJuXXnrJOI5jDh061G21d6e6ujojyVRUVBhjPh4Tj8djVqxYkWhTU1NjJJnKykpjzMf/M+ByuUwwGEy0WbJkifH5fCYSiXTvC+hG/fr1M0899RRjdAaNjY1m2LBhZu3atebLX/5yIqQYq67Toy73RaNRVVVVqbS0NLHN5XKptLRUlZWVKazMHvv27VMwGGw3Rn6/X+PHj0+MUWVlpXJycjR27NhEm9LSUrlcLm3ZsqXba+4OoVBI0v9fnLiqqkqxWKzdOI0YMUIlJSXtxmn06NEqKChItCkrK1M4HNbu3bu7sfru0dbWpuXLl6upqUmBQIAxOoPy8nJNnTq13ZhIvJ+6Uo9aYLa+vl5tbW3t/iVLUkFBgd55550UVWWXYDAoSWcco5P7gsGg8vPz2+13u93Kzc1NtOlN4vG45s6dq6uvvlqjRo2S9PEYpKenKycnp13b08fpTON4cl9vUV1drUAgoJaWFmVlZWnlypUaOXKkdu7cyRidYvny5XrjjTe0bdu2T+zj/dR1elRIAeejvLxcu3bt0qZNm1JdipWGDx+unTt3KhQK6ZlnntHMmTNVUVGR6rKscvDgQc2ZM0dr165Vnz59Ul3OP5Qedbmvf//+SktL+8SMmdraWhUWFqaoKrucHIdPG6PCwkLV1dW129/a2qrjx4/3unGcPXu2Vq9erQ0bNmjgwIGJ7YWFhYpGo2poaGjX/vRxOtM4ntzXW6Snp2vo0KEaM2aMFi9erMsuu0yPPfYYY3SKqqoq1dXV6YorrpDb7Zbb7VZFRYUef/xxud1uFRQUMFZdpEeFVHp6usaMGaN169YltsXjca1bt06BQCCFldlj8ODBKiwsbDdG4XBYW7ZsSYxRIBBQQ0ODqqqqEm3Wr1+veDyu8ePHd3vNXcEYo9mzZ2vlypVav369Bg8e3G7/mDFj5PF42o3Tnj17dODAgXbjVF1d3S7Q165dK5/Pp5EjR3bPC0mBeDyuSCTCGJ1i0qRJqq6u1s6dOxOPsWPHasaMGYl/Zqy6SKpnbpyr5cuXG6/Xa37zm9+Yt99+29x+++0mJyen3YyZ3q6xsdHs2LHD7Nixw0gy//Vf/2V27Nhh/va3vxljPp6CnpOTY5577jnz1ltvmW9961tnnIJ++eWXmy1btphNmzaZYcOG9aop6LNmzTJ+v99s3LjRHDlyJPH46KOPEm3uuOMOU1JSYtavX2+2b99uAoGACQQCif0npwxPnjzZ7Ny50/zpT38yAwYM6FVThu+9915TUVFh9u3bZ9566y1z7733GsdxzCuvvGKMYYw+zamz+4xhrLpKjwspY4x54oknTElJiUlPTzfjxo0zr7/+eqpL6lYbNmwwkj7xmDlzpjHm42noP/rRj0xBQYHxer1m0qRJZs+ePe36OHbsmJk+fbrJysoyPp/PfPe73zWNjY0peDVd40zjI8ksXbo00aa5udl8//vfN/369TOZmZnm29/+tjly5Ei7fvbv32+mTJliMjIyTP/+/c3dd99tYrFYN7+arvO9733PXHjhhSY9Pd0MGDDATJo0KRFQxjBGn+b0kGKsuga36gAAWKtHfScFAPjHQkgBAKxFSAEArEVIAQCsRUgBAKxFSAEArEVIAQCsRUgBAKxFSAEArEVIAQCsRUgBAKxFSAEArPX/ADQp22pQdY6UAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-230.40490074762405"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第7章-DQN算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
